'''
@Author: ysz
@Date: 2019-12-23 15:36:18
@LastEditors  : ysz
@LastEditTime : 2020-01-13 15:26:49
@Description: 爬取二手房信息
'''
import json
import requests
import re
import random
import time
import datetime
from lxml import etree
import os
import os.path

from mysqlhelp import MysqlHelp
from get_user_agent import User_Agent_Help
from common_object import Baixing

myhelp=MysqlHelp()
my_useragent=User_Agent_Help()

class Secondary_House(Baixing):

    #获取详情页信息
    def get_detail_page(self,url):
        print('-----'*20)
        headers={
                'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
                'Accept-Encoding': 'gzip, deflate, br',
                'Accept-Language': 'zh-CN,zh;q=0.9',
                'Cache-Control': 'max-age=0',
                # 'Connection': 'keep-alive',
                'Cookie': 'suid=3807722599; __trackId=157708634659878; _ga=GA1.2.1771081173.1577086347; _gid=GA1.2.2007515902.1577086347; __admx_track_id=vfJpS1IJYFYMR3fbVh_ZGg; __admx_track_id.sig=Czr8QkwjeddFn8A7chX1oS7OPlQ; __s=atsjf95rrifs47m7eav25r1f93; _auth_redirect=https%3A%2F%2Fshenyang.baixing.com%2Fershoufang%2Fa2056054238.html%3Ffrom%3Dvip; seo_ad_test=1; __city=shenyang; Hm_lvt_5a727f1b4acc5725516637e03b07d3d2=1577167250,1577173005,1577173789,1577173801; _gat=1; Hm_lpvt_5a727f1b4acc5725516637e03b07d3d2=1577174963; __sense_session_pv=14',
                'Host': 'shenyang.baixing.com',
                'Referer': 'https://shenyang.baixing.com/ershoufang/?page=2',
                'Sec-Fetch-Mode': 'navigate',
                'Sec-Fetch-Site': 'same-origin',
                'Sec-Fetch-User': '?1',
                'Upgrade-Insecure-Requests': '1',
                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36',
            }
        # try:
        response=requests.get(url=url,headers=headers)
        html_ele=etree.HTML(response.text)

        #所在页面信息

        #标题
        title=html_ele.xpath('//div[@class="viewad-title"]/h1/text()')[0].strip()

        #总价
        total=html_ele.xpath('//div[@class="viewad-actions"]/span[1]/text()')[0].strip().replace('万元','')

        response=response.text
        #房屋面积
        area=re.findall("class='viewad-meta2-item '.*?面积：.*?<label .*?>(.*?)</label",response,re.S)[0].replace('平米','')

        unit=float(total)/float(area)*10000
        #装修情况
        if re.findall("class='viewad-meta2-item '.*?装修情况：.*?<label .*?>(.*?)</label",response,re.S):
            decorate=re.findall("class='viewad-meta2-item '.*?装修情况：.*?<label .*?>(.*?)</label",response,re.S)[0]
        else:
            decorate='未知'
        #是否需要装修
        if decorate=='毛坯':
            isdecorate=0
        else:
            isdecorate=1

        #房屋朝向
        if re.findall("class='viewad-meta2-item '.*?房间朝向：.*?<label .*?>(.*?)</label",response,re.S):
            direction=re.findall("class='viewad-meta2-item '.*?房间朝向：.*?<label .*?>(.*?)</label",response,re.S)[0]
        else:
            direction='未知'

        #所属小区
        if re.findall("class='viewad-meta2-item '.*?小区名：.*?<label .*?>(.*?)</label",response,re.S):
            xq=re.findall("class='viewad-meta2-item '.*?小区名：.*?<label .*?>(.*?)</label",response,re.S)[0]
        else:
            xq='未知'

        #具体位置
        if "具体地点：" in  html_ele.xpath('//div[@class="viewad-meta2-item "]/label[1]/text()'):
            wz=re.findall("class='viewad-meta2-item '.*?具体地点：.*?<label .*?>(.*?)</label",response,re.S)[0]
        else:
            wz='未知'


        # #获取主要数据来源
        if html_ele.xpath('//div[@class="poster-detail"]/a/text()'):
            person_name=html_ele.xpath('//div[@class="poster-detail"]/a/text()')[0]
        else:
            person_name='未知'
        #联系人电话
        if not html_ele.xpath('//p[@id="mobileNumber"]/strong/text()'):
            return 
        phone=html_ele.xpath('//p[@id="mobileNumber"]/strong/text()')[0]
        

        #所在楼层
        if re.findall("class='viewad-meta2-item '.*?楼层：.*?<label .*?>(.*?)</label",response,re.S):
            co_floor=re.findall("class='viewad-meta2-item '.*?楼层：.*?<label .*?>(.*?)</label",response,re.S)[0]
            total_layer=co_floor.split('/')[-1].replace('层','')
            floor=co_floor.split('/')[0]
            floor=self.judge_scope(floor,int(total_layer))
        else:
            floor=0
            total_layer=0


        #产权年限
        # limit_year=page.xpath('//ul[@class="general-item-right"][1]/li[3]/span[2]/text()')[0].replace('年','')
        if "建筑年代：" in  html_ele.xpath('//div[@class="viewad-meta2-item "]/label[1]/text()'):
            nj=re.findall("class='viewad-meta2-item '.*?建筑年代：.*?<label .*?>(.*?)</label",response,re.S)[0].replace('年','')
        else:
            nj=0

        #房屋描述
        #  房屋简介
        if html_ele.xpath('//div[@class="viewad-text"]/text()'):
            desc=html_ele.xpath('//div[@class="viewad-text"]/text()')[0]
        else:
            desc=html_ele.xpath('//div[@class="viewad-text"]/p/text()')[0]

        #获取该房源的所有相关图片url
        if html_ele.xpath('//div[@class="featured-height"]/div/a/@style'):
            imglist=html_ele.xpath('//div[@class="featured-height"]/div/a/@style')
            pattern=re.compile('url\((.*?)\)')
            imglist=pattern.findall(str(imglist))
        else:
            imglist=html_ele.xpath('//div[@class="full-height"]/div/img/@src')
            print(imglist)
        # imglist=[re.sub('/',"\/",i) for i in imglist]
        imgs=json.dumps(imglist)
        print(imgs)

        #房源的url的id
        about_id=url.split('?')[0].split('/')[-1].split('.')[0].replace('a','')
        # 房源信息的来源在58同城上的url地址
        about_url=url
        #爬取的时间
        create_time=int(time.time())
        #状态
        status=1
        print(myhelp.query_data(title=title,area=area,xq=xq,phone=phone,house_type='secondary'),'能不能往数据库存储')
        if myhelp.query_data(title=title,area=area,xq=xq,phone=phone,house_type='secondary'):
            sql='insert into db_housesecond_sh(`unit`,owner_name,about_id,about_url,title,area,total,direction,layer,total_layer,isdecorate,decorate,`desc`,xq,wz,`status`,create_time,imgs,owner_phone,nj) values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'
            data=[unit,person_name,about_id,about_url,title,area,total,direction,floor,total_layer,isdecorate,decorate,desc,xq,wz,status,create_time,imgs,phone,nj]
            house_id=myhelp.execute_sql(sql,data)
            print(house_id,'房源id')
                # print('开始下载图片--------------------------------------------------------')
                # print(imglist,'图片url列表',type(imglist))
                # for img_url in imglist:
                #     create_time=int(time.time())
                #     time.sleep(random.randint(10,20))
                #     pic_path=self.down_img_pic(img_url,url,'secondary')
                #     sql='insert into db_picture2(path,status,create_time,housesecond_id) values(%s,%s,%s,%s)'
                #     data=[pic_path,1,create_time,house_id]
                #     myhelp.execute_sql(sql,data)
                # print('图片下载完成')

            # # print('本次循环的信息：')
            # # print('-----'*20)
            # # #字段设置
            # print(about_id,'房源id')
            # print(about_url,'房源url')
            # print(title,'标题')#--
            # print(area,'面积')
            # print(total,'总价')
            # print(direction,'房屋朝向')
            # print(floor,'楼层')
            # print(total_layer,'总楼层')
            # print(isdecorate,'是否装修')
            # print(decorate,'装修')
            # print(desc,'房屋描述')
            # print(xq,'小区')
            # print(wz,'位置')
            # print(status,'房屋状态')
            # print(create_time,'爬取房屋时间戳')
            # print(imglist,'图片url列表')
            # print(phone,'业主电话')
            # print(nj,'年建')
            # print('---------'*20)
            # print()

        # except:
            # time.sleep(10)
            # self.get_detail_page(url)

#程序的入口  
if __name__ == "__main__":
    # path='./百姓_二手房/download'
    # if not os.path.exists(path):
    #     os.mkdir(path)
    # user_agent=my_useragent.get_useragent()
    url='https://shenyang.baixing.com/ershoufang/?grfy=1&page={}'
    my=Secondary_House(url)
    my.all_page_list()
    # url='https://shenyang.baixing.com/ershoufang/a2078286984.html?from=regular'
    # my.get_detail_page(url)

